package cn.edu.suda.ada.worker;

import house.AnJuKe;
import house.FiveEight;
import house.GanJi;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Random;
import java.util.Scanner;
import java.util.logging.Logger;

import org.apache.http.client.ClientProtocolException;

import cn.edu.suda.ada.fetcher.PageFetcher;
import cn.edu.suda.ada.handler.ContentHandler;
import cn.edu.suda.ada.model.FetchedPage;
import cn.edu.suda.ada.model.Proxy;
import cn.edu.suda.ada.model.SpiderParams;
import cn.edu.suda.ada.parser.AnJuKeHouseParser;
import cn.edu.suda.ada.parser.AnJuKeUrlParser;
import cn.edu.suda.ada.parser.GanJiUrlParser;
import cn.edu.suda.ada.queue.StaticUrlQueue;
import cn.edu.suda.ada.queue.UrlQueue;
import cn.edu.suda.ada.sorage.DataStorage;




public class AnJuKeWorker implements Runnable{
	private static final Logger Log = Logger.getLogger(AnJuKeWorker.class.getName());
//	private static ArrayList<FiveEight> fiveEights=new ArrayList<FiveEight>();
	
	private static UrlQueue urlQueueLevelOne=new UrlQueue();
	private UrlQueue urlQueueLevelTwo=new UrlQueue();
	
//	private PageFetcher fetcher;
//	private ContentHandler handler;
	private AnJuKeHouseParser info;
	private DataStorage store;
	private int threadIndex;
	private String city;
	private int proxyIndex=0;
	
	Random random = new Random();
//	private int max=12;
	private String host="zu.anjuke.com";
	
	public AnJuKeWorker(int threadIndex,String city){
		this.threadIndex = threadIndex;
//		this.fetcher = new PageFetcher();
//		this.handler = new ContentHandler();
		this.info=new AnJuKeHouseParser();
		this.store = new DataStorage(city);
		this.city=city;
		this.host=city+"."+this.host;
//		StaticUrlQueue.addElement("http://su.ganji.com/fang1/1351782251x.htm");
	}
	
	/**
	 * 设置一级页面队列
	 * @method
	 */
	public static void setUrlQueueLevelOne(UrlQueue urlQueueLevelOne) {
		AnJuKeWorker.urlQueueLevelOne = urlQueueLevelOne;
	}
	
	@Override
	public void run() {
		// 登录
		
		
		// 当待抓取URL队列不为空时，执行爬取任务
		// 注： 当队列内容为空时，也不爬取任务已经结束了
		//     因为有可能是UrlQueue暂时空，其他worker线程还没有将新的URL放入队列
		//	        所以，这里可以做个等待时间，再进行抓取（二次机会）
		while(!urlQueueLevelOne.isEmpty()){
			// 从待抓取队列中拿URL
			System.out.println("这是线程"+threadIndex);
			
			String url=urlQueueLevelOne.outElement();//拿出一张页面列表
			PageFetcher fetcher=new PageFetcher();
			FetchedPage page;
			
//			System.out.println("这是线程"+threadIndex);
			
			try {
//				page = fetcher.getHTML(url);
				page=fetcher.getHTMLbyProxy(url, Proxy.hostName[proxyIndex], Proxy.port[proxyIndex],host);
				
//				System.out.println("page:"+page.getContent());
				
				if(null==page.getContent()||page.getStatusCode()==403){
					System.out.println("页面状态代码："+page.getStatusCode());
					proxyIndex=random.nextInt(Proxy.proxyNum);
					Log.info("Spider-" + threadIndex + ": switch IP to "+proxyIndex);
					urlQueueLevelOne.addFirstElement(url);
					continue;
				}
				
				AnJuKeUrlParser urlParser=new AnJuKeUrlParser();
//				urlQueueLevelTwo.clear();
				urlQueueLevelTwo=urlParser.parse(page);
				
				int num=urlQueueLevelTwo.getUrlQueue().size();
				System.out.println("获取到URLs个数:"+num);
				System.out.println(urlQueueLevelTwo.getUrlQueue());
				int j=1;
				
				if(urlQueueLevelTwo.isEmpty()){
					proxyIndex=random.nextInt(Proxy.proxyNum);
					Log.info("Spider-" + threadIndex + ": switch IP to "+proxyIndex);
					urlQueueLevelOne.addFirstElement(url);
					continue;
				}
				
				while(!urlQueueLevelTwo.isEmpty()){
					int k=urlQueueLevelTwo.getUrlQueue().size()-1;
					System.out.println("线程"+threadIndex+"爬第"+j+"个URL，还剩"+k+"个URL");
					j++;
					
					String url2=urlQueueLevelTwo.outElement();
					PageFetcher fetcher2=new PageFetcher();
//					FetchedPage page2=fetcher2.getHTML(url2);
					FetchedPage page2=fetcher2.getHTMLbyProxy(url2, Proxy.hostName[proxyIndex], Proxy.port[proxyIndex],host);
					if(null!=page.getContent()&&page2.getStatusCode()==200){
						try {
							AnJuKe targetData = info.parse(page2);
							if(null==targetData.title||""==targetData.title||"".equals(targetData.title)){
								proxyIndex=random.nextInt(Proxy.proxyNum);
								Log.info("Spider-" + threadIndex + ": switch IP to "+proxyIndex);
								continue;
							}
							System.out.println(page2.getStatusCode());
//							targetData.show();
							store.store(targetData,city);
						} catch (Exception e) {
							// TODO: handle exception
							System.err.println(e);
							continue;
						}
					}else {
						proxyIndex=random.nextInt(Proxy.proxyNum);
						Log.info("Spider-" + threadIndex + ": switch IP to "+proxyIndex);
						continue;
					}
				}
			} catch (ClientProtocolException e1) {
				// TODO Auto-generated catch block
				e1.printStackTrace();
			} catch (IOException e1) {
				// TODO Auto-generated catch block
				e1.printStackTrace();
			}
			
			// delay
			try {
				Thread.sleep(SpiderParams.DEYLAY_TIME);
			} 
			catch (InterruptedException e) {
				e.printStackTrace();
			}
		}
		
		Log.info("Spider-" + threadIndex + ": stop...");
	}
}
